{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It works on a Windows 10 PC which has an old GPU: GeForce GTX 550 Ti.\n",
    "\n",
    "Simply run the following command (not using docker) in the root of this project:\n",
    "\n",
    "```\n",
    "jupyter notebook\n",
    "```\n",
    "\n",
    "Also run the following in a separate terminal:\n",
    "\n",
    "```\n",
    "tensorboard --logdir=./dl/logs --host=127.0.0.1 --port=8889\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow-gpu in c:\\coding\\anaconda3\\lib\\site-packages (1.8.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorflow-gpu) (1.1.0)\n",
      "Requirement already satisfied: protobuf>=3.4.0 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorflow-gpu) (3.6.0)\n",
      "Requirement already satisfied: six>=1.10.0 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorflow-gpu) (1.11.0)\n",
      "Requirement already satisfied: absl-py>=0.1.6 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorflow-gpu) (0.2.2)\n",
      "Requirement already satisfied: astor>=0.6.0 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorflow-gpu) (0.6.2)\n",
      "Requirement already satisfied: wheel>=0.26 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorflow-gpu) (0.31.1)\n",
      "Requirement already satisfied: tensorboard<1.9.0,>=1.8.0 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorflow-gpu) (1.8.0)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorflow-gpu) (1.12.1)\n",
      "Requirement already satisfied: numpy>=1.13.3 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorflow-gpu) (1.14.3)\n",
      "Requirement already satisfied: gast>=0.2.0 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorflow-gpu) (0.2.0)\n",
      "Requirement already satisfied: setuptools in c:\\coding\\anaconda3\\lib\\site-packages (from protobuf>=3.4.0->tensorflow-gpu) (39.1.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorboard<1.9.0,>=1.8.0->tensorflow-gpu) (2.6.11)\n",
      "Requirement already satisfied: werkzeug>=0.11.10 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorboard<1.9.0,>=1.8.0->tensorflow-gpu) (0.14.1)\n",
      "Requirement already satisfied: bleach==1.5.0 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorboard<1.9.0,>=1.8.0->tensorflow-gpu) (1.5.0)\n",
      "Requirement already satisfied: html5lib==0.9999999 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorboard<1.9.0,>=1.8.0->tensorflow-gpu) (0.9999999)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "distributed 1.21.8 requires msgpack, which is not installed.\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow-gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
      "env: CUDA_VISIBLE_DEVICES=0,1\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
    "%env CUDA_VISIBLE_DEVICES=0,1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PCI_BUS_ID\n",
      "0,1\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "print(os.environ[\"CUDA_DEVICE_ORDER\"])\n",
    "print(os.environ[\"CUDA_VISIBLE_DEVICES\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\coding\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[name: \"/device:CPU:0\"\n",
       " device_type: \"CPU\"\n",
       " memory_limit: 268435456\n",
       " locality {\n",
       " }\n",
       " incarnation: 8034297251726998311]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "device_lib.list_local_devices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.test.gpu_device_name()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From https://mxnet.incubator.apache.org/install/index.html :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvcc: NVIDIA (R) Cuda compiler driver\n",
      "Copyright (c) 2005-2017 NVIDIA Corporation\n",
      "Built on Fri_Sep__1_21:08:32_Central_Daylight_Time_2017\n",
      "Cuda compilation tools, release 9.0, V9.0.176\n"
     ]
    }
   ],
   "source": [
    "!nvcc --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: mxnet-cu90 in c:\\coding\\anaconda3\\lib\\site-packages (1.2.0)\n",
      "Requirement already satisfied: requests in c:\\coding\\anaconda3\\lib\\site-packages (from mxnet-cu90) (2.18.4)\n",
      "Requirement already satisfied: numpy in c:\\coding\\anaconda3\\lib\\site-packages (from mxnet-cu90) (1.14.3)\n",
      "Requirement already satisfied: graphviz in c:\\coding\\anaconda3\\lib\\site-packages (from mxnet-cu90) (0.8.3)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\\coding\\anaconda3\\lib\\site-packages (from requests->mxnet-cu90) (3.0.4)\n",
      "Requirement already satisfied: idna<2.7,>=2.5 in c:\\coding\\anaconda3\\lib\\site-packages (from requests->mxnet-cu90) (2.6)\n",
      "Requirement already satisfied: urllib3<1.23,>=1.21.1 in c:\\coding\\anaconda3\\lib\\site-packages (from requests->mxnet-cu90) (1.22)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\coding\\anaconda3\\lib\\site-packages (from requests->mxnet-cu90) (2018.4.16)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "distributed 1.21.8 requires msgpack, which is not installed.\n"
     ]
    }
   ],
   "source": [
    "!pip install mxnet-cu90"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "timestamp, name, pci.bus_id, driver_version, pstate, pcie.link.gen.max, pcie.link.gen.current, temperature.gpu, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]\n",
      "2018/06/29 09:51:58.754, GeForce GTX 550 Ti, 00000000:01:00.0, 385.54, P0, [Not Supported], [Not Supported], 42, [Not Supported], [Not Supported], 1024 MiB, 448 MiB, 576 MiB\n"
     ]
    }
   ],
   "source": [
    "# From https://stackoverflow.com/questions/49076092/is-there-a-way-to-check-if-mxnet-uses-my-gpu/49079940#49079940\n",
    "# https://developer.download.nvidia.com/compute/DCGM/docs/nvidia-smi-367.38.pdf\n",
    "!\"C:\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi\" --query-gpu=timestamp,name,pci.bus_id,driver_version,pstate,pcie.link.gen.max,pcie.link.gen.current,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used --format=csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mxnet as mx \n",
    "def gpu_device(gpu_number=0):\n",
    "    try:\n",
    "        _ = mx.nd.array([1, 2, 3], ctx=mx.gpu(gpu_number))\n",
    "    except mx.MXNetError:\n",
    "        return None\n",
    "    return mx.gpu(gpu_number)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "gpu(0)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpu_device()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "gpu(0)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mx.gpu(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From https://gluon.mxnet.io/chapter03_deep-neural-networks/mlp-gluon.html :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import mxnet as mx\n",
    "from mxnet import nd, autograd, gluon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_ctx = mx.cpu()\n",
    "model_ctx = mx.cpu()\n",
    "# model_ctx = mx.gpu(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "num_inputs = 784\n",
    "num_outputs = 10\n",
    "num_examples = 60000\n",
    "def transform(data, label):\n",
    "    return data.astype(np.float32)/255, label.astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = mx.gluon.data.DataLoader(mx.gluon.data.vision.MNIST(train=True, transform=transform),\n",
    "                                      batch_size, shuffle=True)\n",
    "test_data = mx.gluon.data.DataLoader(mx.gluon.data.vision.MNIST(train=False, transform=transform),\n",
    "                                     batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP(gluon.Block):\n",
    "    def __init__(self, **kwargs):\n",
    "        super(MLP, self).__init__(**kwargs)\n",
    "        with self.name_scope():\n",
    "            self.dense0 = gluon.nn.Dense(64)\n",
    "            self.dense1 = gluon.nn.Dense(64)\n",
    "            self.dense2 = gluon.nn.Dense(10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = nd.relu(self.dense0(x))\n",
    "        x = nd.relu(self.dense1(x))\n",
    "        x = self.dense2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = MLP()\n",
    "net.collect_params().initialize(mx.init.Normal(sigma=.01), ctx=model_ctx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[[-5.2642502e-04 -4.8494569e-04 -9.1017238e-05 -1.0700601e-03\n",
       "   9.5340359e-04  1.2931204e-03 -3.8861975e-04 -6.4619188e-04\n",
       "   1.3646495e-04 -1.7153830e-03]]\n",
       "<NDArray 1x10 @cpu(0)>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = nd.ones((1,784))\n",
    "net(data.as_in_context(model_ctx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hidden Representation 1: \n",
      "[[0.         0.         0.0257028  0.41763663 0.         0.\n",
      "  0.         0.         0.         0.03712562 0.16054314 0.35507876\n",
      "  0.         0.12578698 0.         0.         0.         0.30374664\n",
      "  0.292567   0.35357708 0.         0.07809136 0.21969806 0.2177984\n",
      "  0.         0.3457912  0.13206203 0.01624641 0.27534354 0.22952288\n",
      "  0.2202207  0.         0.00258669 0.06395139 0.68015635 0.\n",
      "  0.         0.         0.1652459  0.18695295 0.25243065 0.01728743\n",
      "  0.06471729 0.         0.         0.2552151  0.         0.\n",
      "  0.03300378 0.33107045 0.6453747  0.04547642 0.         0.\n",
      "  0.         0.19542485 0.02424754 0.         0.         0.04300808\n",
      "  0.16542053 0.13203493 0.         0.        ]]\n",
      "<NDArray 1x64 @cpu(0)>\n",
      "Hidden Representation 2: \n",
      "[[0.0000000e+00 0.0000000e+00 4.8457514e-03 0.0000000e+00 2.4975553e-02\n",
      "  0.0000000e+00 9.2384806e-03 1.1846514e-02 0.0000000e+00 1.5087268e-02\n",
      "  0.0000000e+00 1.3427198e-02 1.6015759e-02 0.0000000e+00 0.0000000e+00\n",
      "  0.0000000e+00 0.0000000e+00 0.0000000e+00 2.7162414e-02 4.1979598e-05\n",
      "  0.0000000e+00 1.8946800e-02 3.0578913e-03 0.0000000e+00 0.0000000e+00\n",
      "  2.7754948e-02 7.5642066e-04 0.0000000e+00 0.0000000e+00 1.9757828e-02\n",
      "  1.7670706e-02 0.0000000e+00 4.0669916e-03 1.0265570e-02 7.5005908e-03\n",
      "  1.5555882e-02 0.0000000e+00 0.0000000e+00 0.0000000e+00 2.8156085e-02\n",
      "  0.0000000e+00 0.0000000e+00 2.0807199e-02 0.0000000e+00 0.0000000e+00\n",
      "  0.0000000e+00 5.2651879e-04 0.0000000e+00 0.0000000e+00 3.6671013e-02\n",
      "  1.6886523e-02 0.0000000e+00 0.0000000e+00 0.0000000e+00 0.0000000e+00\n",
      "  1.5089142e-02 1.0638590e-02 9.0155248e-03 1.8627236e-02 1.4041221e-02\n",
      "  0.0000000e+00 0.0000000e+00 0.0000000e+00 1.2555162e-02]]\n",
      "<NDArray 1x64 @cpu(0)>\n",
      "Network output: \n",
      "[[-1.1785791e-03  1.9014490e-04  8.1118196e-04 -3.8255830e-04\n",
      "   4.7956721e-04 -1.2719276e-04  3.3852040e-05 -2.3284566e-04\n",
      "   7.1805023e-04  1.1753932e-03]]\n",
      "<NDArray 1x10 @cpu(0)>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\n",
       "[[-1.1785791e-03  1.9014490e-04  8.1118196e-04 -3.8255830e-04\n",
       "   4.7956721e-04 -1.2719276e-04  3.3852040e-05 -2.3284566e-04\n",
       "   7.1805023e-04  1.1753932e-03]]\n",
       "<NDArray 1x10 @cpu(0)>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class MLP(gluon.Block):\n",
    "    def __init__(self, **kwargs):\n",
    "        super(MLP, self).__init__(**kwargs)\n",
    "        with self.name_scope():\n",
    "            self.dense0 = gluon.nn.Dense(64, activation=\"relu\")\n",
    "            self.dense1 = gluon.nn.Dense(64, activation=\"relu\")\n",
    "            self.dense2 = gluon.nn.Dense(10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.dense0(x)\n",
    "        print(\"Hidden Representation 1: %s\" % x)\n",
    "        x = self.dense1(x)\n",
    "        print(\"Hidden Representation 2: %s\" % x)\n",
    "        x = self.dense2(x)\n",
    "        print(\"Network output: %s\" % x)\n",
    "        return x\n",
    "\n",
    "net = MLP()\n",
    "net.collect_params().initialize(mx.init.Normal(sigma=.01), ctx=model_ctx)\n",
    "net(data.as_in_context(model_ctx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_hidden = 64\n",
    "net = gluon.nn.HybridSequential()\n",
    "with net.name_scope():\n",
    "    net.add(gluon.nn.Dense(num_hidden, activation=\"relu\"))\n",
    "    net.add(gluon.nn.Dense(num_hidden, activation=\"relu\"))\n",
    "    net.add(gluon.nn.Dense(num_outputs))\n",
    "net.hybridize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "net.collect_params().initialize(mx.init.Normal(sigma=.1), ctx=model_ctx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': .01})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_accuracy(data_iterator, net):\n",
    "    acc = mx.metric.Accuracy()\n",
    "    for i, (data, label) in enumerate(data_iterator):\n",
    "        data = data.as_in_context(model_ctx).reshape((-1, 784))\n",
    "        label = label.as_in_context(model_ctx)\n",
    "        output = net(data)\n",
    "        predictions = nd.argmax(output, axis=1)\n",
    "        acc.update(preds=predictions, labels=label)\n",
    "    return acc.get()[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: mxboard in c:\\coding\\anaconda3\\lib\\site-packages (0.1.0)\n",
      "Requirement already satisfied: protobuf>=3.0.0 in c:\\coding\\anaconda3\\lib\\site-packages (from mxboard) (3.6.0)\n",
      "Requirement already satisfied: Pillow in c:\\coding\\anaconda3\\lib\\site-packages (from mxboard) (5.1.0)\n",
      "Requirement already satisfied: six in c:\\coding\\anaconda3\\lib\\site-packages (from mxboard) (1.11.0)\n",
      "Requirement already satisfied: numpy in c:\\coding\\anaconda3\\lib\\site-packages (from mxboard) (1.14.3)\n",
      "Requirement already satisfied: setuptools in c:\\coding\\anaconda3\\lib\\site-packages (from protobuf>=3.0.0->mxboard) (39.1.0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "distributed 1.21.8 requires msgpack, which is not installed.\n"
     ]
    }
   ],
   "source": [
    "!pip install mxboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorboard in c:\\coding\\anaconda3\\lib\\site-packages (1.8.0)\n",
      "Requirement already satisfied: html5lib==0.9999999 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorboard) (0.9999999)\n",
      "Requirement already satisfied: bleach==1.5.0 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorboard) (1.5.0)\n",
      "Requirement already satisfied: protobuf>=3.4.0 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorboard) (3.6.0)\n",
      "Requirement already satisfied: werkzeug>=0.11.10 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorboard) (0.14.1)\n",
      "Requirement already satisfied: six>=1.10.0 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorboard) (1.11.0)\n",
      "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in c:\\coding\\anaconda3\\lib\\site-packages (from tensorboard) (0.31.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorboard) (2.6.11)\n",
      "Requirement already satisfied: numpy>=1.12.0 in c:\\coding\\anaconda3\\lib\\site-packages (from tensorboard) (1.14.3)\n",
      "Requirement already satisfied: setuptools in c:\\coding\\anaconda3\\lib\\site-packages (from protobuf>=3.4.0->tensorboard) (39.1.0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "distributed 1.21.8 requires msgpack, which is not installed.\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mxboard import SummaryWriter\n",
    "sw = SummaryWriter(logdir='logs', flush_secs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0. Loss: 1.2356886094411215, Train_acc 0.8395166666666667, Test_acc 0.8474\n",
      "Epoch 1. Loss: 0.46565542084376016, Train_acc 0.88465, Test_acc 0.8912\n",
      "Epoch 2. Loss: 0.3715192502895991, Train_acc 0.901, Test_acc 0.9029\n",
      "Epoch 3. Loss: 0.32939287207126616, Train_acc 0.91, Test_acc 0.9112\n",
      "Epoch 4. Loss: 0.30172612600326537, Train_acc 0.91625, Test_acc 0.9184\n",
      "Epoch 5. Loss: 0.2804558823386828, Train_acc 0.9217333333333333, Test_acc 0.921\n",
      "Epoch 6. Loss: 0.2626380964756012, Train_acc 0.9262666666666667, Test_acc 0.9252\n",
      "Epoch 7. Loss: 0.24708882774909338, Train_acc 0.9300333333333334, Test_acc 0.9296\n",
      "Epoch 8. Loss: 0.23393845278819403, Train_acc 0.9341833333333334, Test_acc 0.9332\n",
      "Epoch 9. Loss: 0.22262413431803385, Train_acc 0.93545, Test_acc 0.9352\n"
     ]
    }
   ],
   "source": [
    "epochs = 10\n",
    "smoothing_constant = .01\n",
    "\n",
    "# collect parameter names for logging the gradients of parameters in each epoch\n",
    "params = net.collect_params()\n",
    "param_names = params.keys()\n",
    "global_step = 0\n",
    "\n",
    "for e in range(epochs):\n",
    "    cumulative_loss = 0\n",
    "    for i, (data, label) in enumerate(train_data):\n",
    "        data = data.as_in_context(model_ctx).reshape((-1, 784))\n",
    "        label = label.as_in_context(model_ctx)\n",
    "        with autograd.record():\n",
    "            output = net(data)\n",
    "            loss = softmax_cross_entropy(output, label)\n",
    " \n",
    "        sw.add_scalar(tag='cross_entropy', value=loss.mean().asscalar(), global_step=global_step)\n",
    "        if i == 0:\n",
    "            sw.add_image('minist_first_minibatch', data.reshape((batch_size, 1, 28, 28)), e)\n",
    "        if e == 0:\n",
    "            sw.add_graph(net)\n",
    "        grads = [i.grad() for i in net.collect_params().values()]\n",
    "        for i, name in enumerate(param_names):\n",
    "            sw.add_histogram(tag=name, values=grads[i], global_step=e, bins=1000)\n",
    "\n",
    "        global_step += 1\n",
    "        loss.backward()\n",
    "        trainer.step(data.shape[0])\n",
    "        cumulative_loss += nd.sum(loss).asscalar()\n",
    "\n",
    "        \n",
    "    \n",
    "    test_accuracy = evaluate_accuracy(test_data, net)\n",
    "    train_accuracy = evaluate_accuracy(train_data, net)\n",
    "    sw.add_scalar(tag='accuracy_curves', value=('train_acc', train_accuracy), global_step=e)\n",
    "    sw.add_scalar(tag='accuracy_curves', value=('valid_acc', test_accuracy), global_step=e)\n",
    "    print(\"Epoch %s. Loss: %s, Train_acc %s, Test_acc %s\" %\n",
    "          (e, cumulative_loss/num_examples, train_accuracy, test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](images/mx_01_acc.png)\n",
    "![](images/mx_02_loss.png)\n",
    "![](images/mx_03_img.png)\n",
    "![](images/mx_04_net.png)\n",
    "![](images/mx_05_dist.png)\n",
    "![](images/mx_06_hist.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
